import json
from os import makedirs
from os.path import exists

import requests
from bs4 import BeautifulSoup
from gevent import os


def req_get_bf(url):
    return BeautifulSoup(requests.get(url, timeout=180).text, 'html.parser')


def getTitle(url):
    title = url.split('/')
    if title[-1] == "":
        return (title[-2])
    else:
        return title[-1]


dirName = 'tags'
filename = 'top_tag.txt'

isAuto = False


def jumpUrl(url, i_num, i_total):
    bf = req_get_bf(url)
    title = getTitle(url)
    empty = []
    thttags = bf.find_all("div", "tht-tags")
    if thttags:
        for tags in thttags:
            if tags.string and (tags.string != ''):
                empty.append(tags.string)
        txt = dirName + '/' + filename
        exist_file = exists(txt)
        if not exists(dirName):
            makedirs(dirName)
        with open(txt, 'a+', encoding='utf-8') as f:
            if exist_file:
                f.write(",\n")
            data_dict = {
                'title': title,
                'i_num': i_num,
                'i_total': i_total,
                'tags': empty
            }
            json_data = json.dumps(data_dict, indent=4)
            f.write(json_data)
            print(json_data)
            f.close()


def realFetchTag(tagUrl, i_num, i_total):
    url = "https://top-hashtags.com" + tagUrl
    jumpUrl(url, i_num, i_total)


def checkTopTag(page):
    url = 'https://top-hashtags.com/instagram/'
    if page > 100:
        url += str(page) + '/'
    print(url)
    bf = req_get_bf(url)
    i_row = bf.find_all('li', 'i-row')
    if i_row:
        for row in i_row:
            i_num = row.find('div', 'i-num')
            i_total = row.find('div', 'i-total')

            link_a = row.find('a')
            href = str(link_a.get('href'))
            if href.startswith('/hashtag'):
                realFetchTag(href, int(i_num.string), i_total.string)

    if isAuto:
        page += 100
        checkTopTag(page)
    else:
        print("\n ---- Tag All Fetch End ----")


if __name__ == '__main__':
    checkTopTag(0)
